#ifndef TEXTENTITYRECOGNITION_TOKENIZER_H
#define TEXTENTITYRECOGNITION_TOKENIZER_H
#include <string>
#include <vector>
#include <map>
using namespace std;

namespace localvit {
    struct Node {
        map <int8_t, Node> sons;
        int index = -1;
    };

    class Tokenizer {
    public:
        Tokenizer();

        ~Tokenizer();

        void Insert(string word);

        void LoadVocab(const char *labelData, size_t labelLength);

        int Find(string word);

        bool Find(int8_t ch, Node *node);

        void CHWordDeal(string token, vector<int> &inpus_id, vector <string> &tokens,
                        vector<int> &offset, int &point);

        void ENWordDeal(string token, vector<int> &inputs_id, vector <string> &tokens,
                        vector<int> &offsets, int &point);

        void EncodeText(const std::string &text, vector<int> &inputs_id, vector <string> &tokens,
                        vector<int> &offset);

    private:
        Node *vocabs_ = nullptr;
        vector <string> words_;
        int pad_id_ = 0;
        int unk_id_ = 100;
        int cls_id_ = 101;
        int sep_id_ = 102;
    };
}
#endif //TEXTENTITYRECOGNITION_TOKENIZER_H
